#include "stdafx.h"
#include "FeatSelector.h"
#include "SOPMI.h"
#include <math.h>
//bool LookUpInDict(sentiDict &bigDict,wordRateMap &fisherTable,std::vector<CString> &posList,std::vector<CString> &negList)
//{
//    wordRateMap::reverse_iterator riter=fisherTable.rbegin();
//    for(;riter!=fisherTable.rend();riter++)
//    {
//        sentiDict::iterator iter=bigDict.find(riter->second);
//        if(iter!=bigDict.end())
//        {
//            if(iter->second==SPOS)
//                posList.push_back(riter->second);
//            else
//                negList.push_back(riter->second);
//        }
//    }
//    return true;
//}
bool writeDict(std::vector<CString> &words,CString &filename)
{
    CStdioFile file;
    if(!file.Open(filename,CStdioFile::modeWrite|CStdioFile::modeCreate))
        return false;
    for(int i=0;i<(int)words.size();i++)
        file.WriteString(words[i]+"\n");
    file.Close();
    return true;
}
////////////////////////////////////////////////////////
CSelector::CSelector(const CString& workDir,int dim,featType type,CString keyword)
{
    m_workDir		= workDir;
    m_dim           = dim;
	m_type		    = type;
	m_keyword		= keyword;
	m_classNum		= getCount(workDir+CString("\\config\\trainClasses.dat"));
}
CSelector::CSelector()
{
}
bool CSelector::Select()
{
    _construct();
    _calculate();
    return true;
}
bool CSelector::_construct()
{
	CStdioFile iFile;
	CString filename=m_workDir+"\\config\\featureCountList"+m_keyword+".dat";
	if(!iFile.Open(filename,CStdioFile::modeRead))
		return false;
	ULONGLONG len=iFile.GetLength();
	while(iFile.GetPosition()!=len)
	{
		CString tmp;
		iFile.ReadString(tmp);
		CString name;
		//int v1,v2;
		std::vector<int> countList;
		for(int i=0;i<2;i++)
		{
			int nStart=tmp.ReverseFind(' ');
			CString value_str=tmp.Mid(nStart);
			countList.push_back(atoi(value_str.GetBuffer()));
			tmp=tmp.Mid(0,nStart);
		}
		name.Format("%s",tmp);
		m_wCMap.insert(wordCountPair(name,countList));
	}
	return true;
}
bool CSelector::_calculate()
{   
    wordCountMap::iterator iter=m_wCMap.begin();
	rateWordMap rWMap;
    int totalAll=0;
    std::vector<double> Pc;
    for(int i=0;i<(int)m_classNum.size();i++)
        totalAll+=m_classNum[i];
    for(int i=0;i<(int)m_classNum.size();i++)
        Pc.push_back((m_classNum[i]*1.0)/totalAll);
    for(;iter!=m_wCMap.end();iter++)
    {
        double rate=0;
		int totalT=0;
		for(int i=0;i< (int)iter->second.size();i++)
		{
				   totalT+=iter->second[i];
		}
		for(int i=0;i < (int)m_classNum.size();i++)
		{
				double p_c,p_t,p_n_c,p_n_t;
				double p_c_t,p_c_n_t;
				double p_n_c_t,p_n_c_n_t;
				
				p_c= (m_classNum[i]*1.0)/totalAll;
				p_t= (totalT*1.0)/totalAll;
				p_c_t= (iter->second[i]*1.0)/totalAll;

				p_n_c = 1 - p_c;
				p_n_t = 1 - p_t;
				p_n_c_t = p_t - p_c_t;
				p_c_n_t = p_c - p_c_t;
				p_n_c_n_t = p_n_t - p_c_n_t;

				switch(m_type)
				{
				case ftIG:
					if (fabs(p_c*p_n_t) > dZero && fabs(p_c_n_t) > dZero)
						rate += p_c_n_t * log(p_c_n_t/(p_c * p_n_t));
					if (fabs(p_c*p_t) > dZero && fabs(p_c_t) > dZero)
						rate += p_c_t * log(p_c_t/(p_c * p_t));
					break;
				case ftCHI:
					if (fabs(p_n_c*p_t*p_n_t) > dZero)
						rate += totalAll * (p_c_t * p_n_c_n_t - p_n_c_t * p_c_n_t) * (p_c_t * p_n_c_n_t - p_n_c_t * p_c_n_t) / ( p_t * p_n_c * p_n_t);
					break;
				case ftMI:
					if (fabs(p_c*p_t) > dZero && fabs(p_c_t) > dZero)
						rate += p_c * log(p_c_t/(p_c * p_t));
					break;
				case ftFISHER:
					break;
				}
		}
		
		if((int)rWMap.size()>=m_dim)
		{
			if(rate>rWMap.begin()->first)
				rWMap.erase(rWMap.begin());
			else
				continue;
		}
		rWMap.insert(rateWordPair(rate,iter->first));
    }
    rateWordMap::const_reverse_iterator riter=rWMap.rbegin();
    rateWordMap::const_reverse_iterator rend =rWMap.rend();
    CStdioFile fileOutput;
    if(!fileOutput.Open(m_workDir+CString("\\config\\featureRateList"+m_keyword+".dat"),
						CStdioFile::modeWrite|CStdioFile::modeCreate))
        return false;
	int index=0;
    for(;riter!=rend;riter++)
    {
        CString str;
        str.Format("%s %f\n",riter->second,riter->first);
        fileOutput.WriteString(str);
		m_wRMap.insert(wordRatePair(riter->second,riter->first));
		m_wNMap.insert(wordSingleCountPair(riter->second,index++));
    }
    fileOutput.Close();
    return true;
}
bool CSelector::_present(CString catogory)
{
	CFileFind fFolder;
	BOOL folderFlg=
		 fFolder.FindFile(m_workDir+"\\"+catogory+"\\pre"+m_keyword+"\\*.*");
	int index=1;
	int totalAll=0;
	for(int i=0;i<(int)m_classNum.size();i++)
        totalAll+=m_classNum[i];
	CStdioFile oFile,oFileList;
	oFile.Open(m_workDir+"\\config\\"+catogory+m_keyword+".dat",
			   CStdioFile::modeWrite|CStdioFile::modeCreate);
	oFileList.Open(m_workDir+"\\config\\fileList"+catogory+m_keyword+".dat",
			   CStdioFile::modeWrite|CStdioFile::modeCreate);
	while(folderFlg)
	{
		folderFlg=fFolder.FindNextFile();
		if(fFolder.IsDirectory()&&!fFolder.IsDots())
		{
			CFileFind fTxt;
			BOOL txtFlg=fTxt.FindFile(fFolder.GetFilePath()+CString("\\*.*"));
			while(txtFlg)
			{
				txtFlg=fTxt.FindNextFile();
				if(!fTxt.IsDirectory()&&!fTxt.IsDots())
				{
					CStdioFile iFile;
					featRateMap frM;
					iFile.Open(fTxt.GetFilePath(),CStdioFile::modeRead);
					ULONGLONG len=iFile.GetLength();
					CString str;
					str.Format("%d",index);
					while(iFile.GetPosition()!=len)
					{
						CString tmp;
						iFile.ReadString(tmp);						
						int position=tmp.ReverseFind(' ');
						int tf=atoi(tmp.Mid(position));
						CString name=tmp.Mid(0,position);
						wordSingleCountMap::iterator iter=m_wNMap.find(name);
						if(iter!=m_wNMap.end())
						{
							std::vector<int> countList=m_wCMap.find(name)->second;
							int deno=0;
							for(int i=0;i<(int)countList.size();i++)
							{
								deno+=countList[i];
							}
							double tf_idf=log(1.0+tf)*log((double)(totalAll*1.0)/deno);
							frM.insert(featRatePair(iter->second+1,tf_idf));			
						}
					}
					iFile.Close();
					double total=0.0;

					for(featRateMap::iterator iter=frM.begin();iter!=frM.end();iter++)
					{
						total+=pow(iter->second,2);
					}
					total=sqrt(total);
					for(featRateMap::iterator iter=frM.begin();iter!=frM.end();iter++)
					{
						CString present;
						present.Format(" %d:%f",iter->first,iter->second/total);
						str+=present;
					}
					if(str.GetLength()>1)//this text has some features
					{
						str+=CString("\n");
						oFile.WriteString(str);
						CString fileContent;
						fileContent.Format("%s %d\n",fTxt.GetFileName(),index-1);
						oFileList.WriteString(fileContent);
					}					
				}
			}
			fTxt.Close();
			index++;
		}	
	}
	fFolder.Close();
	//listFile.Close();
	oFile.Close();
	oFileList.Close();
	return true;
}
bool CSelector::Present()
{
	return _present("train")&&
		   _present("test");
}